{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import configparser\n",
    "import requests\n",
    "import logging\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import re\n",
    "import sys\n",
    "import argparse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loding configuration\n",
    "config = configparser.ConfigParser()\n",
    "config.read('config.ini')\n",
    "website_config = config['WEBSITE']\n",
    "logging_config = config['LOGGING']\n",
    "\n",
    "#metadata of the data to be crawled\n",
    "data_columns = [\n",
    "\"questionID\",\n",
    "\"questionTitle\",\n",
    "\"questionText\",\n",
    "\"questionLink\",\n",
    "\"topic\",\n",
    "\"therapistName\",\n",
    "\"therapistTitle\",\n",
    "\"therapistURL\",\n",
    "\"answerText\",\n",
    "\"upvotes\",\n",
    "\"views\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions to Crawl Questions and then answers from CounselChat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_HTML_from_string(rawText):\n",
    "    cleanr = re.compile('<.*?>')\n",
    "    cleantext = re.sub(cleanr, '', rawText)\n",
    "    return cleantext\n",
    "\n",
    "def find_question_item_url(questionItem):\n",
    "    return website_config['WEBSITE_MAIN_URL'] + questionItem.find(\"a\", {\"class\": \"question-title\"}).attrs['href']\n",
    "\n",
    "def get_question_urls_for_a_topic(topic):\n",
    "    logging.info('Enter get_question_urls_for_a_topic') \n",
    "    result = []\n",
    "    count = 1\n",
    "    while (count==1 or questionItems != []):\n",
    "        url = website_config['WEBSITE_MAIN_URL'] + website_config['TOPICS_URL'] + \"/\" + topic + \"?page=\" + str(count)\n",
    "        logging.debug(\"Topic URL: \" + url) \n",
    "        page = requests.get(url)    \n",
    "        soup = BeautifulSoup(page.text, 'html.parser')\n",
    "        questionItems = soup.find(\"div\", {\"class\": \"list-group list-question-group\"}).find_all(\"div\", {\"class\": \"item-question\"})                         \n",
    "        questionUrls = list(map(find_question_item_url,questionItems))    \n",
    "        logging.debug(\"Question URLs\")\n",
    "        logging.debug(questionUrls)         \n",
    "        result+=questionUrls         \n",
    "        count += 1\n",
    "\n",
    "    logging.debug(\"Result of get_question_urls_for_a_topic\")\n",
    "    logging.debug(result)    \n",
    "    logging.info('Exit get_question_urls_for_a_topic') \n",
    "    return result\n",
    "\n",
    "def write_list_into_file(filename,listItems):\n",
    "    logging.info('Enter write_list_into_file')     \n",
    "    with open(filename, \"a\") as filehandle:\n",
    "        for listItem in listItems:\n",
    "            filehandle.write('%s\\n' % listItem)\n",
    "    logging.info('Exit write_list_into_file')\n",
    "\n",
    "def read_list_from_file(filename):    \n",
    "    logging.info('Enter read_list_from_file')     \n",
    "    url_list = []    \n",
    "    with open(filename, 'r') as filehandle:\n",
    "        for line in filehandle:\n",
    "            # remove linebreak which is the last character of the string\n",
    "            currentPlace = line[:-1]            \n",
    "            url_list.append(currentPlace)            \n",
    "    logging.info('Exit read_list_from_file')\n",
    "    return url_list    \n",
    "        \n",
    "def crawl_questions():    \n",
    "    logging.info('Enter crawl_questions') \n",
    "    topics = website_config[\"TOPICS\"].split(\",\")\n",
    "    logging.debug(\"Topics: \"+str(topics))\n",
    "    for topic in topics:\n",
    "        url_list = get_question_urls_for_a_topic(topic)\n",
    "        write_list_into_file(\"questions/\"+topic+\".txt\",url_list)\n",
    "    logging.info('Exit crawl_questions') \n",
    "\n",
    "\n",
    "def get_string_from_list(listItems):\n",
    "    result = \"\"\n",
    "    for listItem in listItems:\n",
    "        result += listItem.text\n",
    "    return result\n",
    "\n",
    "def get_question_and_answer_details(topic,questionId,url):\n",
    "    logging.info('Enter get_question_and_answer_details') \n",
    "    result = []    \n",
    "    page = requests.get(url)    \n",
    "    soup = BeautifulSoup(page.text, 'html.parser')\n",
    "    content = soup.find(\"body\").find(\"div\",{\"id\":\"content\"})\n",
    "    \n",
    "    # question\n",
    "    questionDetailHTML = content.find(\"div\",{\"class\":\"row\"})\n",
    "    questionTitle = remove_HTML_from_string(questionDetailHTML.find(\"h1\",{\"class\":\"page-title\"}).text) \n",
    "    logging.debug(\"QuestionTitle: \"+ questionTitle)        \n",
    "    questionTextHTML = questionDetailHTML.find(\"div\",{\"class\":\"page-description\"})\n",
    "    paragraphs = questionTextHTML.findAll(\"p\")    \n",
    "    questionText = remove_HTML_from_string(get_string_from_list(paragraphs))\n",
    "    logging.debug(\"QuestionText: \"+ questionText)  \n",
    "    # answers\n",
    "    answersDetails = content.find_all(\"div\", {\"class\": \"item-answer\"})\n",
    "    logging.debug(\"answerDetails: \") \n",
    "    logging.debug(answersDetails) \n",
    "\n",
    "\n",
    "    for answerDetail in answersDetails:\n",
    "        # question and answer row    \n",
    "        dataItem = []        \n",
    "        # 1. questionID\n",
    "        dataItem.append(questionId)\n",
    "        # 2. questionTitle \n",
    "        dataItem.append(questionTitle)\n",
    "        # 3. questionText \n",
    "        dataItem.append(questionText)\n",
    "        # 4. questionLink\n",
    "        dataItem.append(url)\n",
    "        # 5. topic\n",
    "        dataItem.append(topic)                \n",
    "        therapistDetail = answerDetail.find(\"div\",{\"class\":\"therapist-summary\"}).find(\"div\",{\"class\":\"name-title\"})\n",
    "        # 6. therapistName\n",
    "        dataItem.append(therapistDetail.find(\"a\",{\"class\":\"name\"}).text)\n",
    "        # 7. therapistTitle\n",
    "        dataItem.append(therapistDetail.find(\"div\",{\"class\":\"title\"}).text)\n",
    "        # 8. therapistURL\n",
    "        dataItem.append(website_config['WEBSITE_MAIN_URL'] + therapistDetail.find(\"a\",{\"class\":\"name\"}).attrs['ng-href'])\n",
    "        answerParagraphsList = answerDetail.find(\"div\",{\"class\":\"description\"}).findAll(\"p\") \n",
    "        # # 9. answerText\n",
    "        answerText = remove_HTML_from_string(get_string_from_list(answerParagraphsList))\n",
    "\n",
    "        #removing extraspaces, single and double quotes        \n",
    "        cleanText = ' '.join(answerText.split()).replace(\"'\",\"\").replace(\"\\\"\",\"\")                        \n",
    "        dataItem.append(cleanText)\n",
    "\n",
    "        actions = answerDetail.find(\"div\",{\"class\":\"actions\"}).findAll(\"li\")\n",
    "        # 10. upvotes\n",
    "        dataItem.append(int(actions[0].find(\"a\").attrs['ng-init'].split(\"=\")[1].strip()))\n",
    "        # # 11. views\n",
    "        dataItem.append(int(actions[1].find(\"span\").text.split(' ')[0]))        \n",
    "        result.append(dataItem)\n",
    "\n",
    "    logging.debug(\"Result of get_question_and_answer_details: \")      \n",
    "    logging.debug(result)      \n",
    "    logging.info('Exit get_question_and_answer_details') \n",
    "    return result\n",
    "    \n",
    "\n",
    "def crawl_answers():\n",
    "    logging.info('Enter crawl_answers')       \n",
    "    topics = website_config[\"TOPICS\"].split(\",\")\n",
    "    logging.debug(\"Topics: \"+str(topics))\n",
    "    topic_count = 1\n",
    "    for topic in topics:    \n",
    "        logging.debug(\"Crawl for Topic: \" + topic)\n",
    "        print (\"Topic:  \"+str(topic_count)+\"/\"+str(len(topics)))        \n",
    "        logging.info(\"Topic:  \"+str(topic_count)+\"/\"+str(len(topics))) \n",
    "        df = pd.DataFrame(columns=data_columns)        \n",
    "        url_list = read_list_from_file(\"questions/\"+topic+\".txt\")\n",
    "        count = 1\n",
    "        for url in url_list:                     \n",
    "            logging.info('Crwaling URL: '+str(count)+\"/\"+str(len(url_list)))      \n",
    "            logging.info(\"Crawl for URL: \" + url)                    \n",
    "            dataItems = get_question_and_answer_details(topic,count,url)\n",
    "            for dataItem in dataItems:\n",
    "                df.loc[len(df)] = dataItem                \n",
    "            count+=1\n",
    "        logging.debug('Output dataframe of a topic') \n",
    "        logging.debug(df) \n",
    "        df.to_excel(\"output/\"+topic+\".xlsx\")  \n",
    "        topic_count+=1\n",
    "    logging.info('Exit crawl_answers') \n",
    "\n",
    "def crawl_questions_and_answers():\n",
    "    crawl_questions() \n",
    "    crawl_answers()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T06:56:39.282414Z",
     "start_time": "2020-05-04T06:56:39.198748Z"
    }
   },
   "outputs": [],
   "source": [
    "def main():    \n",
    "    #logging\n",
    "    logging.basicConfig(filename=logging_config[\"LOG_FILE_NAME\"],level=logging.INFO,filemode=\"w\")\n",
    "    logging.info('Application Started')\n",
    "    # crawl_questions() \n",
    "    crawl_answers()\n",
    "    # crawl_questions_and_answers()\n",
    "    logging.info('Application Ended')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
